#!/usr/bin/env python3
"""
系统诊断工具 - 快速检测和解决系统卡住问题
"""

import os
import sys
import json
import time
import threading
import traceback
import psutil
from datetime import datetime
from typing import Dict, Any, List, Optional
from pathlib import Path

from logger_config import hr_logger
from timeout_monitor import timeout_monitor

class SystemDiagnostic:
    """系统诊断器"""
    
    def __init__(self):
        self.diagnostic_data = {}
        self.last_check_time = None
    
    def quick_health_check(self) -> Dict[str, Any]:
        """快速健康检查"""
        hr_logger.info("开始系统健康检查")
        
        health_data = {
            "timestamp": datetime.now().isoformat(),
            "overall_status": "unknown",
            "checks": {}
        }
        
        # 1. 内存检查
        memory_status = self._check_memory()
        health_data["checks"]["memory"] = memory_status
        
        # 2. CPU检查  
        cpu_status = self._check_cpu()
        health_data["checks"]["cpu"] = cpu_status
        
        # 3. 线程检查
        thread_status = self._check_threads()
        health_data["checks"]["threads"] = thread_status
        
        # 4. 网络连接检查
        network_status = self._check_network()
        health_data["checks"]["network"] = network_status
        
        # 5. 文件系统检查
        filesystem_status = self._check_filesystem()
        health_data["checks"]["filesystem"] = filesystem_status
        
        # 6. API密钥检查
        api_status = self._check_api_keys()
        health_data["checks"]["api_keys"] = api_status
        
        # 7. 超时监控检查
        timeout_status = self._check_timeout_monitor()
        health_data["checks"]["timeout_monitor"] = timeout_status
        
        # 8. 数据管理器检查
        data_status = self._check_data_manager()
        health_data["checks"]["data_manager"] = data_status
        
        # 综合状态评估
        health_data["overall_status"] = self._evaluate_overall_status(health_data["checks"])
        
        return health_data
    
    def _check_memory(self) -> Dict[str, Any]:
        """检查内存使用情况"""
        try:
            memory = psutil.virtual_memory()
            process = psutil.Process(os.getpid())
            
            status = {
                "status": "ok",
                "system_memory_percent": memory.percent,
                "system_available_gb": round(memory.available / (1024**3), 2),
                "process_memory_percent": process.memory_percent(),
                "process_memory_mb": round(process.memory_info().rss / (1024**2), 2),
                "warnings": []
            }
            
            # 检查内存使用过高
            if memory.percent > 90:
                status["status"] = "critical"
                status["warnings"].append("系统内存使用过高")
            elif memory.percent > 80:
                status["status"] = "warning" 
                status["warnings"].append("系统内存使用较高")
            
            if process.memory_percent() > 20:
                status["status"] = "warning"
                status["warnings"].append("进程内存使用较高")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法获取内存信息"]
            }
    
    def _check_cpu(self) -> Dict[str, Any]:
        """检查CPU使用情况"""
        try:
            cpu_percent = psutil.cpu_percent(interval=1)
            process = psutil.Process(os.getpid())
            
            status = {
                "status": "ok",
                "system_cpu_percent": cpu_percent,
                "process_cpu_percent": process.cpu_percent(),
                "cpu_count": psutil.cpu_count(),
                "warnings": []
            }
            
            if cpu_percent > 90:
                status["status"] = "critical"
                status["warnings"].append("系统CPU使用过高")
            elif cpu_percent > 80:
                status["status"] = "warning"
                status["warnings"].append("系统CPU使用较高")
                
            return status
            
        except Exception as e:
            return {
                "status": "error", 
                "error": str(e),
                "warnings": ["无法获取CPU信息"]
            }
    
    def _check_threads(self) -> Dict[str, Any]:
        """检查线程状态"""
        try:
            all_threads = threading.enumerate()
            daemon_threads = [t for t in all_threads if t.daemon]
            non_daemon_threads = [t for t in all_threads if not t.daemon]
            
            status = {
                "status": "ok",
                "total_threads": len(all_threads),
                "daemon_threads": len(daemon_threads),
                "non_daemon_threads": len(non_daemon_threads),
                "thread_details": [],
                "warnings": []
            }
            
            # 详细线程信息
            for thread in all_threads:
                thread_info = {
                    "name": thread.name,
                    "daemon": thread.daemon,
                    "alive": thread.is_alive(),
                    "ident": thread.ident
                }
                status["thread_details"].append(thread_info)
            
            # 检查线程数量异常
            if len(all_threads) > 50:
                status["status"] = "warning"
                status["warnings"].append("线程数量过多")
            
            # 检查是否有卡住的线程
            active_count = threading.active_count()
            if active_count != len(all_threads):
                status["status"] = "warning"
                status["warnings"].append("线程计数不一致")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法获取线程信息"]
            }
    
    def _check_network(self) -> Dict[str, Any]:
        """检查网络连接"""
        try:
            process = psutil.Process(os.getpid())
            connections = process.connections()
            
            status = {
                "status": "ok",
                "connection_count": len(connections),
                "established_connections": 0,
                "listening_ports": [],
                "warnings": []
            }
            
            for conn in connections:
                if conn.status == 'ESTABLISHED':
                    status["established_connections"] += 1
                elif conn.status == 'LISTEN':
                    status["listening_ports"].append(conn.laddr.port)
            
            # 检查连接数过多
            if len(connections) > 100:
                status["status"] = "warning"
                status["warnings"].append("网络连接数过多")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法获取网络信息"]
            }
    
    def _check_filesystem(self) -> Dict[str, Any]:
        """检查文件系统"""
        try:
            current_dir = Path(__file__).parent
            disk_usage = psutil.disk_usage(str(current_dir))
            
            status = {
                "status": "ok",
                "disk_total_gb": round(disk_usage.total / (1024**3), 2),
                "disk_used_gb": round(disk_usage.used / (1024**3), 2),
                "disk_free_gb": round(disk_usage.free / (1024**3), 2),
                "disk_usage_percent": round((disk_usage.used / disk_usage.total) * 100, 2),
                "log_directory_exists": (current_dir / "logs").exists(),
                "warnings": []
            }
            
            # 检查磁盘空间
            if status["disk_usage_percent"] > 95:
                status["status"] = "critical"
                status["warnings"].append("磁盘空间不足")
            elif status["disk_usage_percent"] > 90:
                status["status"] = "warning"
                status["warnings"].append("磁盘空间较少")
            
            # 检查日志目录
            if not status["log_directory_exists"]:
                status["warnings"].append("日志目录不存在")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法获取文件系统信息"]
            }
    
    def _check_api_keys(self) -> Dict[str, Any]:
        """检查API密钥配置"""
        status = {
            "status": "ok",
            "dashscope_key_exists": bool(os.getenv("DASHSCOPE_API_KEY")),
            "openai_key_exists": bool(os.getenv("OPENAI_API_KEY")),
            "warnings": []
        }
        
        if not status["dashscope_key_exists"] and not status["openai_key_exists"]:
            status["status"] = "critical"
            status["warnings"].append("未找到任何API密钥")
        elif not status["dashscope_key_exists"]:
            status["warnings"].append("未找到DashScope API密钥")
        elif not status["openai_key_exists"]:
            status["warnings"].append("未找到OpenAI API密钥")
            
        return status
    
    def _check_timeout_monitor(self) -> Dict[str, Any]:
        """检查超时监控状态"""
        try:
            timeout_status = timeout_monitor.get_status()
            
            status = {
                "status": "ok",
                "monitoring_active": timeout_status["monitoring_active"],
                "active_operations": timeout_status["active_operations_count"],
                "total_timeouts": timeout_status["total_timeouts"],
                "warnings": []
            }
            
            if not timeout_status["monitoring_active"]:
                status["status"] = "warning"
                status["warnings"].append("超时监控未激活")
            
            if timeout_status["active_operations_count"] > 10:
                status["status"] = "warning"
                status["warnings"].append("活跃操作过多")
            
            if timeout_status["total_timeouts"] > 5:
                status["warnings"].append("历史超时事件较多")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法获取超时监控信息"]
            }
    
    def _check_data_manager(self) -> Dict[str, Any]:
        """检查数据管理器状态"""
        try:
            from data_manager import data_manager
            
            status = {
                "status": "ok",
                "resumes_loaded": len(data_manager.resumes),
                "jobs_loaded": len(data_manager.jobs),
                "warnings": []
            }
            
            if status["resumes_loaded"] == 0:
                status["status"] = "critical"
                status["warnings"].append("简历数据未加载")
            
            if status["jobs_loaded"] == 0:
                status["status"] = "warning"
                status["warnings"].append("岗位数据未加载")
                
            return status
            
        except Exception as e:
            return {
                "status": "error",
                "error": str(e),
                "warnings": ["无法检查数据管理器"]
            }
    
    def _evaluate_overall_status(self, checks: Dict[str, Any]) -> str:
        """评估整体状态"""
        has_critical = any(check.get("status") == "critical" for check in checks.values())
        has_error = any(check.get("status") == "error" for check in checks.values())
        has_warning = any(check.get("status") == "warning" for check in checks.values())
        
        if has_critical or has_error:
            return "critical"
        elif has_warning:
            return "warning"
        else:
            return "healthy"
    
    def diagnose_stuck_request(self, request_context: Dict[str, Any] = None) -> Dict[str, Any]:
        """诊断卡住的请求"""
        hr_logger.info("开始诊断卡住的请求")
        
        diagnosis = {
            "timestamp": datetime.now().isoformat(),
            "request_context": request_context or {},
            "possible_causes": [],
            "recommendations": [],
            "system_snapshot": {}
        }
        
        # 获取系统快照
        health_check = self.quick_health_check()
        diagnosis["system_snapshot"] = health_check
        
        # 分析可能原因
        if health_check["overall_status"] == "critical":
            diagnosis["possible_causes"].append("系统资源严重不足")
            diagnosis["recommendations"].append("检查内存和CPU使用，重启系统")
        
        # 检查超时监控
        timeout_status = timeout_monitor.get_status()
        if timeout_status["active_operations_count"] > 0:
            diagnosis["possible_causes"].append("存在长时间运行的操作")
            diagnosis["recommendations"].append("检查活跃操作，考虑终止超时任务")
            diagnosis["active_operations"] = timeout_status["active_operations"]
        
        # 检查线程状态
        if health_check["checks"]["threads"]["total_threads"] > 20:
            diagnosis["possible_causes"].append("线程数量过多")
            diagnosis["recommendations"].append("检查是否有线程泄露")
        
        # 检查API连接
        if not health_check["checks"]["api_keys"]["dashscope_key_exists"] and \
           not health_check["checks"]["api_keys"]["openai_key_exists"]:
            diagnosis["possible_causes"].append("API密钥配置问题")
            diagnosis["recommendations"].append("检查并设置正确的API密钥")
        
        # 检查网络
        if health_check["checks"]["network"]["connection_count"] > 50:
            diagnosis["possible_causes"].append("网络连接过多")
            diagnosis["recommendations"].append("检查网络连接，可能存在连接泄露")
        
        return diagnosis
    
    def generate_diagnostic_report(self) -> str:
        """生成诊断报告"""
        health_check = self.quick_health_check()
        diagnosis = self.diagnose_stuck_request()
        
        report = f"""
        
================ HR系统诊断报告 ================
时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
整体状态: {health_check['overall_status'].upper()}

🖥️  系统资源:
   - 内存使用: {health_check['checks']['memory']['system_memory_percent']:.1f}%
   - CPU使用: {health_check['checks']['cpu']['system_cpu_percent']:.1f}%
   - 线程数量: {health_check['checks']['threads']['total_threads']}
   - 网络连接: {health_check['checks']['network']['connection_count']}

📊 数据状态:
   - 简历数量: {health_check['checks']['data_manager']['resumes_loaded']}
   - 岗位数量: {health_check['checks']['data_manager']['jobs_loaded']}

🔑 API配置:
   - DashScope: {'✅' if health_check['checks']['api_keys']['dashscope_key_exists'] else '❌'}
   - OpenAI: {'✅' if health_check['checks']['api_keys']['openai_key_exists'] else '❌'}

⏱️  超时监控:
   - 监控状态: {'✅' if health_check['checks']['timeout_monitor']['monitoring_active'] else '❌'}
   - 活跃操作: {health_check['checks']['timeout_monitor']['active_operations']}
   - 历史超时: {health_check['checks']['timeout_monitor']['total_timeouts']}

⚠️  问题诊断:
"""
        
        # 添加警告信息
        all_warnings = []
        for check_name, check_data in health_check['checks'].items():
            warnings = check_data.get('warnings', [])
            for warning in warnings:
                all_warnings.append(f"   - [{check_name}] {warning}")
        
        if all_warnings:
            report += "\n".join(all_warnings)
        else:
            report += "   - 未发现明显问题"
        
        # 添加推荐措施
        if diagnosis['recommendations']:
            report += f"\n\n💡 推荐措施:\n"
            for i, rec in enumerate(diagnosis['recommendations'], 1):
                report += f"   {i}. {rec}\n"
        
        report += f"\n{'='*50}\n"
        
        return report

# 全局诊断实例
system_diagnostic = SystemDiagnostic()

def quick_diagnostic():
    """快速诊断命令"""
    print("🔍 正在进行系统诊断...")
    report = system_diagnostic.generate_diagnostic_report()
    print(report)
    
    # 保存到文件
    log_dir = Path(__file__).parent / "logs"
    log_dir.mkdir(exist_ok=True)
    
    report_file = log_dir / f"diagnostic_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
    with open(report_file, 'w', encoding='utf-8') as f:
        f.write(report)
    
    print(f"📄 诊断报告已保存到: {report_file}")

if __name__ == "__main__":
    # 如果直接运行，执行快速诊断
    quick_diagnostic()












